{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from keras.datasets import reuters\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# parameters for data load\n",
    "num_words = 30000\n",
    "maxlen = 50\n",
    "test_split = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = reuters.load_data(num_words = num_words, maxlen = maxlen, test_split = test_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pad the sequences with zeros \n",
    "# padding parameter is set to 'post' => 0's are appended to end of sequences\n",
    "X_train = pad_sequences(X_train, padding = 'post')\n",
    "X_test = pad_sequences(X_test, padding = 'post')\n",
    "\n",
    "X_train = np.array(X_train).reshape((X_train.shape[0], X_train.shape[1], 1))\n",
    "X_test = np.array(X_test).reshape((X_test.shape[0], X_test.shape[1], 1))\n",
    "\n",
    "y_data = np.concatenate((y_train, y_test))\n",
    "y_data = to_categorical(y_data)\n",
    "y_train = y_data[:1395]\n",
    "y_test = y_data[1395:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Deep RNN\n",
    "- RNNs can be made deep, with multiple layers, like CNNs or MLPs\n",
    "- Beware that RNNs take long to train compared to CNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, LSTM, Activation\n",
    "from keras import optimizers\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def deep_lstm():\n",
    "    model = Sequential()\n",
    "    model.add(LSTM(20, input_shape = (49,1), return_sequences = True))\n",
    "    model.add(LSTM(20, return_sequences = True))\n",
    "    model.add(LSTM(20, return_sequences = True))\n",
    "    model.add(LSTM(20, return_sequences = False))\n",
    "    model.add(Dense(46))\n",
    "    model.add(Activation('softmax'))\n",
    "    \n",
    "    adam = optimizers.Adam(lr = 0.001)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = KerasClassifier(build_fn = deep_lstm, epochs = 200, batch_size = 50, verbose = 1)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "599/599 [==============================] - 0s     \n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "y_test_ = np.argmax(y_test, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.816360601002\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_pred, y_test_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Bidirectional RNN\n",
    "- Bidirectional RNNs consider not only one-way influence of sequence, but also the other way\n",
    "- Actually, they can be thought as building two separate RNNs, and merging them\\\n",
    "<br>\n",
    "<img src=\"http://d3kbpzbmcynnmx.cloudfront.net/wp-content/uploads/2015/09/bidirectional-rnn.png\" style=\"width: 400px\"/>\n",
    "</br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import Bidirectional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bidirectional_lstm():\n",
    "    model = Sequential()\n",
    "    model.add(Bidirectional(LSTM(20, return_sequences = False), input_shape = (49,1)))\n",
    "    model.add(Dense(46))\n",
    "    model.add(Activation('softmax'))\n",
    "    \n",
    "    adam = optimizers.Adam(lr = 0.001)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = KerasClassifier(build_fn = bidirectional_lstm, epochs = 200, batch_size = 50, verbose = 1)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500/599 [========================>.....] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "y_test_ = np.argmax(y_test, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.841402337229\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_pred, y_test_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Deep Bidirectional RNN\n",
    "- Bidirectional RNNs can be stacked\n",
    "\n",
    "<img src=\"http://www.wildml.com/wp-content/uploads/2015/09/Screen-Shot-2015-09-16-at-2.21.51-PM-272x300.png\" style=\"width: 300px\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def deep_bidirectional_lstm():\n",
    "    model = Sequential()\n",
    "    model.add(Bidirectional(LSTM(10, return_sequences = True), input_shape = (49,1)))\n",
    "    model.add(Bidirectional(LSTM(10, return_sequences = True)))\n",
    "    model.add(Bidirectional(LSTM(10, return_sequences = True)))\n",
    "    model.add(Bidirectional(LSTM(10, return_sequences = False)))\n",
    "    model.add(Dense(46))\n",
    "    model.add(Activation('softmax'))\n",
    "    \n",
    "    adam = optimizers.Adam(lr = 0.001)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = KerasClassifier(build_fn = deep_bidirectional_lstm, epochs = 200, batch_size = 50, verbose = 1)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "599/599 [==============================] - 1s     \n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "y_test_ = np.argmax(y_test, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.824707846411\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_pred, y_test_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
